datapath-windows/ovsext/PacketIO.c

   1 /*
   2  * Copyright (c) 2014 VMware, Inc.
   3  *
   4  * Licensed under the Apache License, Version 2.0 (the "License");
   5  * you may not use this file except in compliance with the License.
   6  * You may obtain a copy of the License at:
   7  *
   8  *     http://www.apache.org/licenses/LICENSE-2.0
   9  *
  10  * Unless required by applicable law or agreed to in writing, software
  11  * distributed under the License is distributed on an "AS IS" BASIS,
  12  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  13  * See the License for the specific language governing permissions and
  14  * limitations under the License.
  15  */
  16
  17 /*
  18  * This file contains the implementation of the datapath/forwarding
  19  * functionality of the OVS.
  20  */
  21
  22 #include "precomp.h"
  23 #include "Switch.h"
  24 #include "Vport.h"
  25 #include "NetProto.h"
  26 #include "User.h"
  27 #include "PacketIO.h"
  28 #include "Flow.h"
  29 #include "Event.h"
  30 #include "User.h"
  31
  32 /* Due to an imported header file */
  33 #pragma warning( disable:4505 )
  34
  35 #ifdef OVS_DBG_MOD
  36 #undef OVS_DBG_MOD
  37 #endif
  38 #define OVS_DBG_MOD OVS_DBG_DISPATCH
  39 #include "Debug.h"
  40
  41 extern NDIS_STRING ovsExtGuidUC;
  42 extern NDIS_STRING ovsExtFriendlyNameUC;
  43
  44 static VOID OvsFinalizeCompletionList(OvsCompletionList *completionList);
  45 static VOID OvsCompleteNBLIngress(POVS_SWITCH_CONTEXT switchContext,
  46                     PNET_BUFFER_LIST netBufferLists, ULONG sendCompleteFlags);
  47 static NTSTATUS OvsCreateNewNBLsFromMultipleNBs(
  48                     POVS_SWITCH_CONTEXT switchContext,
  49                     PNET_BUFFER_LIST *curNbl,
  50                     PNET_BUFFER_LIST *nextNbl);
  51
  52 __inline VOID
  53 OvsInitCompletionList(OvsCompletionList *completionList,
  54                       POVS_SWITCH_CONTEXT switchContext,
  55                       ULONG sendCompleteFlags)
  56 {
  57     ASSERT(completionList);
  58     completionList->dropNbl = NULL;
  59     completionList->dropNblNext = &completionList->dropNbl;
  60     completionList->switchContext = switchContext;
  61     completionList->sendCompleteFlags = sendCompleteFlags;
  62 }
  63
  64 /* Utility function used to complete an NBL. */
  65 __inline VOID
  66 OvsAddPktCompletionList(OvsCompletionList *completionList,
  67                         BOOLEAN incoming,
  68                         NDIS_SWITCH_PORT_ID sourcePort,
  69                         PNET_BUFFER_LIST netBufferList,
  70                         UINT32 netBufferListCount,
  71                         PNDIS_STRING filterReason)
  72 {
  73     POVS_BUFFER_CONTEXT ctx;
  74
  75     /* XXX: We handle one NBL at a time. */
  76     ASSERT(netBufferList->Next == NULL);
  77
  78     /* Make sure it has a context. */
  79     ctx = (POVS_BUFFER_CONTEXT)NET_BUFFER_LIST_CONTEXT_DATA_START(netBufferList);
  80     ASSERT(ctx && ctx->magic == OVS_CTX_MAGIC);
  81
  82     completionList->switchContext->NdisSwitchHandlers.ReportFilteredNetBufferLists(
  83         completionList->switchContext->NdisSwitchContext, &ovsExtGuidUC,
  84         &ovsExtFriendlyNameUC, sourcePort,
  85         incoming ? NDIS_SWITCH_REPORT_FILTERED_NBL_FLAGS_IS_INCOMING : 0,
  86         netBufferListCount, netBufferList, filterReason);
  87
  88     *completionList->dropNblNext = netBufferList;
  89     completionList->dropNblNext = &netBufferList->Next;
  90     ASSERT(completionList->dropNbl);
  91 }
  92
  93 static __inline VOID
  94 OvsReportNBLIngressError(POVS_SWITCH_CONTEXT switchContext,
  95                          PNET_BUFFER_LIST nblList,
  96                          PNDIS_STRING filterReason,
  97                          NDIS_STATUS error)
  98 {
  99     PNET_BUFFER_LIST nbl = nblList;
 100     while (nbl) {
 101         PNDIS_SWITCH_FORWARDING_DETAIL_NET_BUFFER_LIST_INFO fwdDetail;
 102         fwdDetail = NET_BUFFER_LIST_SWITCH_FORWARDING_DETAIL(nbl);
 103
 104         nbl->Status = error;
 105
 106         /* This can be optimized by batching NBL's from the same
 107          * SourcePortId. */
 108         switchContext->NdisSwitchHandlers.ReportFilteredNetBufferLists(
 109             switchContext->NdisSwitchContext, &ovsExtGuidUC,
 110             &ovsExtFriendlyNameUC, fwdDetail->SourcePortId,
 111             NDIS_SWITCH_REPORT_FILTERED_NBL_FLAGS_IS_INCOMING,
 112             1 /*Nbl count.*/, nbl, filterReason);
 113
 114         nbl = NET_BUFFER_LIST_NEXT_NBL(nbl);
 115     }
 116 }
 117
 118 static __inline ULONG
 119 OvsGetSendCompleteFlags(ULONG sendFlags)
 120 {
 121     BOOLEAN dispatch, sameSource;
 122     ULONG sendCompleteFlags;
 123
 124     dispatch = NDIS_TEST_SEND_AT_DISPATCH_LEVEL(sendFlags);
 125     sendCompleteFlags = (dispatch ?
 126                         NDIS_SEND_COMPLETE_FLAGS_DISPATCH_LEVEL : 0);
 127     sameSource = NDIS_TEST_SEND_FLAG(sendFlags,
 128                                         NDIS_SEND_FLAGS_SWITCH_SINGLE_SOURCE);
 129     sendCompleteFlags |= (sameSource ?
 130                         NDIS_SEND_COMPLETE_FLAGS_SWITCH_SINGLE_SOURCE : 0);
 131
 132     return sendCompleteFlags;
 133 }
 134
 135 VOID
 136 OvsSendNBLIngress(POVS_SWITCH_CONTEXT switchContext,
 137                   PNET_BUFFER_LIST netBufferLists,
 138                   ULONG sendFlags)
 139 {
 140     if (switchContext->dataFlowState == OvsSwitchPaused) {
 141         /* If a filter module is in the Paused state, the filter driver must not
 142          * originate any send requests for that filter module. If NDIS calls
 143          * FilterSendNetBufferLists, the driver must not call
 144          * NdisFSendNetBufferLists to pass on the data until the driver is
 145          * restarted. The driver should call NdisFSendNetBufferListsComplete
 146          * immediately to complete the send operation. It should set the
 147          * complete status in each NET_BUFFER_LIST structure to
 148          * NDIS_STATUS_PAUSED.
 149          *
 150          * http://msdn.microsoft.com/en-us/library/windows/hardware/
 151          * ff549966(v=vs.85).aspx */
 152         NDIS_STRING filterReason;
 153         ULONG sendCompleteFlags = OvsGetSendCompleteFlags(sendFlags);
 154
 155         RtlInitUnicodeString(&filterReason,
 156                              L"Switch state PAUSED, drop before FSendNBL.");
 157         OvsReportNBLIngressError(switchContext, netBufferLists, &filterReason,
 158                                  NDIS_STATUS_PAUSED);
 159         OvsCompleteNBLIngress(switchContext, netBufferLists,
 160                               sendCompleteFlags);
 161         return;
 162     }
 163
 164     ASSERT(switchContext->dataFlowState == OvsSwitchRunning);
 165
 166     NdisFSendNetBufferLists(switchContext->NdisFilterHandle, netBufferLists,
 167                             NDIS_DEFAULT_PORT_NUMBER, sendFlags);
 168 }
 169
 170 static __inline VOID
 171 OvsStartNBLIngressError(POVS_SWITCH_CONTEXT switchContext,
 172                         PNET_BUFFER_LIST nblList,
 173                         ULONG sendCompleteFlags,
 174                         PNDIS_STRING filterReason,
 175                         NDIS_STATUS error)
 176 {
 177     ASSERT(error);
 178     OvsReportNBLIngressError(switchContext, nblList, filterReason, error);
 179     NdisFSendNetBufferListsComplete(switchContext->NdisFilterHandle, nblList,
 180                                     sendCompleteFlags);
 181 }
 182
 183 static VOID
 184 OvsAppendNativeForwardedPacket(POVS_SWITCH_CONTEXT switchContext,
 185                                PNET_BUFFER_LIST curNbl,
 186                                PNET_BUFFER_LIST *nativeNbls,
 187                                ULONG flags,
 188                                BOOLEAN isRecv)
 189 {
 190     POVS_BUFFER_CONTEXT ctx = { 0 };
 191     NDIS_STRING filterReason;
 192
 193     *nativeNbls = curNbl;
 194     nativeNbls = &(curNbl->Next);
 195
 196     ctx = OvsInitExternalNBLContext(switchContext, curNbl, isRecv);
 197     if (ctx == NULL) {
 198         RtlInitUnicodeString(&filterReason,
 199                              L"Cannot allocate native NBL context.");
 200
 201         OvsStartNBLIngressError(switchContext, curNbl, flags, &filterReason,
 202                                 NDIS_STATUS_RESOURCES);
 203     }
 204 }
 205
 206 static VOID
 207 OvsStartNBLIngress(POVS_SWITCH_CONTEXT switchContext,
 208                    PNET_BUFFER_LIST netBufferLists,
 209                    ULONG SendFlags)
 210 {
 211     NDIS_SWITCH_PORT_ID sourcePort = 0;
 212     NDIS_SWITCH_NIC_INDEX sourceIndex = 0;
 213     PNDIS_SWITCH_FORWARDING_DETAIL_NET_BUFFER_LIST_INFO fwdDetail;
 214     PNET_BUFFER_LIST curNbl = NULL, nextNbl = NULL;
 215     ULONG sendCompleteFlags;
 216     UCHAR dispatch;
 217     LOCK_STATE_EX lockState, dpLockState;
 218     NDIS_STATUS status;
 219     NDIS_STRING filterReason;
 220     LIST_ENTRY missedPackets;
 221     UINT32 num = 0;
 222     OvsCompletionList completionList;
 223 #if (NDIS_SUPPORT_NDIS640)
 224     PNET_BUFFER_LIST nativeForwardedNbls = NULL;
 225     PNET_BUFFER_LIST *nextNativeForwardedNbl = &nativeForwardedNbls;
 226 #endif
 227
 228     dispatch = NDIS_TEST_SEND_AT_DISPATCH_LEVEL(SendFlags)?
 229                                             NDIS_RWL_AT_DISPATCH_LEVEL : 0;
 230     sendCompleteFlags = OvsGetSendCompleteFlags(SendFlags);
 231     SendFlags |= NDIS_SEND_FLAGS_SWITCH_DESTINATION_GROUP;
 232
 233     InitializeListHead(&missedPackets);
 234     OvsInitCompletionList(&completionList, switchContext, sendCompleteFlags);
 235
 236     for (curNbl = netBufferLists; curNbl != NULL; curNbl = nextNbl) {
 237         POVS_VPORT_ENTRY vport;
 238         UINT32 portNo;
 239         OVS_DATAPATH *datapath = &switchContext->datapath;
 240         OVS_PACKET_HDR_INFO layers;
 241         OvsFlowKey key;
 242         UINT64 hash;
 243         PNET_BUFFER curNb;
 244         POVS_BUFFER_CONTEXT ctx;
 245
 246         nextNbl = curNbl->Next;
 247         curNbl->Next = NULL;
 248
 249         fwdDetail = NET_BUFFER_LIST_SWITCH_FORWARDING_DETAIL(curNbl);
 250         sourcePort = fwdDetail->SourcePortId;
 251         sourceIndex = (NDIS_SWITCH_NIC_INDEX)fwdDetail->SourceNicIndex;
 252
 253 #if (NDIS_SUPPORT_NDIS640)
 254         if (fwdDetail->NativeForwardingRequired) {
 255             /* Add current NBL to those that require native forwarding. */
 256             OvsAppendNativeForwardedPacket(
 257                 switchContext,
 258                 curNbl,
 259                 nextNativeForwardedNbl,
 260                 sendCompleteFlags,
 261                 sourcePort == switchContext->virtualExternalPortId);
 262             continue;
 263         }
 264 #endif /* NDIS_SUPPORT_NDIS640 */
 265
 266         ctx = OvsInitExternalNBLContext(switchContext, curNbl,
 267                   sourcePort == switchContext->virtualExternalPortId);
 268         if (ctx == NULL) {
 269             RtlInitUnicodeString(&filterReason,
 270                 L"Cannot allocate external NBL context.");
 271
 272             OvsStartNBLIngressError(switchContext, curNbl,
 273                                     sendCompleteFlags, &filterReason,
 274                                     NDIS_STATUS_RESOURCES);
 275             continue;
 276         }
 277
 278         /* Ethernet Header is a guaranteed safe access. */
 279         curNb = NET_BUFFER_LIST_FIRST_NB(curNbl);
 280         if (curNb->Next != NULL) {
 281             /* Create a NET_BUFFER_LIST for each NET_BUFFER. */
 282             status = OvsCreateNewNBLsFromMultipleNBs(switchContext,
 283                                                      &curNbl,
 284                                                      &nextNbl);
 285             if (!NT_SUCCESS(status)) {
 286                 RtlInitUnicodeString(&filterReason,
 287                                      L"Cannot allocate NBLs with single NB.");
 288
 289                 OvsStartNBLIngressError(switchContext, curNbl,
 290                                         sendCompleteFlags, &filterReason,
 291                                         NDIS_STATUS_RESOURCES);
 292                 continue;
 293             }
 294         }
 295         {
 296             OvsFlow *flow;
 297
 298             /* Take the DispatchLock so none of the VPORTs disconnect while
 299              * we are setting destination ports.
 300              *
 301              * XXX: acquire/release the dispatch lock for a "batch" of packets
 302              * rather than for each packet. */
 303             NdisAcquireRWLockRead(switchContext->dispatchLock, &lockState,
 304                                   dispatch);
 305
 306             vport = OvsFindVportByPortIdAndNicIndex(switchContext, sourcePort,
 307                                                     sourceIndex);
 308             if (vport == NULL || vport->ovsState != OVS_STATE_CONNECTED) {
 309                 RtlInitUnicodeString(&filterReason,
 310                     L"OVS-Cannot forward packet from unknown source port");
 311                 goto dropit;
 312             } else {
 313                 portNo = vport->portNo;
 314             }
 315
 316             vport->stats.rxPackets++;
 317             vport->stats.rxBytes += NET_BUFFER_DATA_LENGTH(curNb);
 318
 319             status = OvsExtractFlow(curNbl, vport->portNo, &key, &layers, NULL);
 320             if (status != NDIS_STATUS_SUCCESS) {
 321                 RtlInitUnicodeString(&filterReason, L"OVS-Flow extract failed");
 322                 goto dropit;
 323             }
 324
 325             ASSERT(KeGetCurrentIrql() == DISPATCH_LEVEL);
 326             OvsAcquireDatapathRead(datapath, &dpLockState, TRUE);
 327
 328             flow = OvsLookupFlow(datapath, &key, &hash, FALSE);
 329             if (flow) {
 330                 OvsFlowUsed(flow, curNbl, &layers);
 331                 datapath->hits++;
 332                 /* If successful, OvsActionsExecute() consumes the NBL.
 333                  * Otherwise, it adds it to the completionList. No need to
 334                  * check the return value. */
 335                 OvsActionsExecute(switchContext, &completionList, curNbl,
 336                                 portNo, SendFlags, &key, &hash, &layers,
 337                                 flow->actions, flow->actionsLen);
 338                 OvsReleaseDatapath(datapath, &dpLockState);
 339                 NdisReleaseRWLock(switchContext->dispatchLock, &lockState);
 340                 continue;
 341             } else {
 342                 OvsReleaseDatapath(datapath, &dpLockState);
 343
 344                 datapath->misses++;
 345                 status = OvsCreateAndAddPackets(NULL, 0, OVS_PACKET_CMD_MISS,
 346                              portNo, &key, curNbl,
 347                              sourcePort == switchContext->virtualExternalPortId,
 348                              &layers, switchContext, &missedPackets, &num);
 349                 if (status == NDIS_STATUS_SUCCESS) {
 350                     /* Complete the packet since it was copied to user
 351                      * buffer. */
 352                     RtlInitUnicodeString(&filterReason,
 353                         L"OVS-Dropped since packet was copied to userspace");
 354                 } else {
 355                     RtlInitUnicodeString(&filterReason,
 356                         L"OVS-Dropped due to failure to queue to userspace");
 357                 }
 358                 goto dropit;
 359             }
 360
 361 dropit:
 362             OvsAddPktCompletionList(&completionList, TRUE, sourcePort, curNbl, 0,
 363                                     &filterReason);
 364             NdisReleaseRWLock(switchContext->dispatchLock, &lockState);
 365         }
 366     }
 367
 368 #if (NDIS_SUPPORT_NDIS640)
 369     if (nativeForwardedNbls) {
 370         /* This is NVGRE encapsulated traffic and is forwarded to NDIS
 371          * in order to be handled by the HNV module. */
 372         OvsSendNBLIngress(switchContext, nativeForwardedNbls, SendFlags);
 373     }
 374 #endif /* NDIS_SUPPORT_NDIS640 */
 375
 376     /* Queue the missed packets. */
 377     OvsQueuePackets(&missedPackets, num);
 378     OvsFinalizeCompletionList(&completionList);
 379 }
 380
 381
 382 /*
 383  * --------------------------------------------------------------------------
 384  * Implements filter driver's FilterSendNetBufferLists Function.
 385  * --------------------------------------------------------------------------
 386  */
 387 VOID
 388 OvsExtSendNBL(NDIS_HANDLE filterModuleContext,
 389               PNET_BUFFER_LIST netBufferLists,
 390               NDIS_PORT_NUMBER portNumber,
 391               ULONG sendFlags)
 392 {
 393     UNREFERENCED_PARAMETER(portNumber);
 394
 395     /* 'filterModuleContext' is the switch context that gets created in the
 396      * AttachHandler. */
 397     POVS_SWITCH_CONTEXT switchContext;
 398     switchContext = (POVS_SWITCH_CONTEXT) filterModuleContext;
 399
 400     if (switchContext->dataFlowState == OvsSwitchPaused) {
 401         NDIS_STRING filterReason;
 402         ULONG sendCompleteFlags = OvsGetSendCompleteFlags(sendFlags);
 403
 404         RtlInitUnicodeString(&filterReason,
 405                              L"Switch state PAUSED, drop on ingress.");
 406         OvsStartNBLIngressError(switchContext, netBufferLists,
 407                                 sendCompleteFlags, &filterReason,
 408                                 NDIS_STATUS_PAUSED);
 409         return;
 410     }
 411
 412     ASSERT(switchContext->dataFlowState == OvsSwitchRunning);
 413
 414     OvsStartNBLIngress(switchContext, netBufferLists, sendFlags);
 415 }
 416
 417 static VOID
 418 OvsCompleteNBLIngress(POVS_SWITCH_CONTEXT switchContext,
 419                       PNET_BUFFER_LIST netBufferLists,
 420                       ULONG sendCompleteFlags)
 421 {
 422     PNET_BUFFER_LIST curNbl = NULL, nextNbl = NULL;
 423     OvsCompletionList newList;
 424
 425     newList.dropNbl = NULL;
 426     newList.dropNblNext = &newList.dropNbl;
 427
 428     for (curNbl = netBufferLists; curNbl != NULL; curNbl = nextNbl) {
 429         nextNbl = curNbl->Next;
 430         curNbl->Next = NULL;
 431
 432         curNbl = OvsCompleteNBL(switchContext, curNbl, TRUE);
 433         if (curNbl != NULL) {
 434             /* NBL originated from the upper layer. */
 435             *newList.dropNblNext = curNbl;
 436             newList.dropNblNext = &curNbl->Next;
 437         }
 438     }
 439
 440     /* Complete the NBL's that were sent by the upper layer. */
 441     if (newList.dropNbl != NULL) {
 442         NdisFSendNetBufferListsComplete(switchContext->NdisFilterHandle, newList.dropNbl,
 443                                         sendCompleteFlags);
 444     }
 445 }
 446
 447
 448 /*
 449  * --------------------------------------------------------------------------
 450  * Implements filter driver's FilterSendNetBufferListsComplete function.
 451  * --------------------------------------------------------------------------
 452  */
 453 VOID
 454 OvsExtSendNBLComplete(NDIS_HANDLE filterModuleContext,
 455                       PNET_BUFFER_LIST netBufferLists,
 456                       ULONG sendCompleteFlags)
 457 {
 458     OvsCompleteNBLIngress((POVS_SWITCH_CONTEXT)filterModuleContext,
 459                           netBufferLists, sendCompleteFlags);
 460 }
 461
 462
 463 VOID
 464 OvsFinalizeCompletionList(OvsCompletionList *completionList)
 465 {
 466     if (completionList->dropNbl != NULL) {
 467         OvsCompleteNBLIngress(completionList->switchContext,
 468                               completionList->dropNbl,
 469                               completionList->sendCompleteFlags);
 470
 471         completionList->dropNbl = NULL;
 472         completionList->dropNblNext = &completionList->dropNbl;
 473     }
 474 }
 475
 476 /*
 477  * --------------------------------------------------------------------------
 478  * Implements filter driver's FilterCancelSendNetBufferLists function.
 479  *
 480  * "If a filter driver specifies a FilterSendNetBufferLists function and it
 481  * queues send requests, it must also specify a
 482  * FilterCancelSendNetBufferLists function."
 483  *
 484  * http://msdn.microsoft.com/en-us/library/windows/hardware/
 485  * ff549966(v=vs.85).aspx
 486  * --------------------------------------------------------------------------
 487  */
 488 VOID
 489 OvsExtCancelSendNBL(NDIS_HANDLE filterModuleContext,
 490                     PVOID CancelId)
 491 {
 492     UNREFERENCED_PARAMETER(filterModuleContext);
 493     UNREFERENCED_PARAMETER(CancelId);
 494
 495     /* All send requests get completed synchronously, so there is no need to
 496      * implement this callback. */
 497 }
 498
 499 static NTSTATUS
 500 OvsCreateNewNBLsFromMultipleNBs(POVS_SWITCH_CONTEXT switchContext,
 501                                 PNET_BUFFER_LIST *curNbl,
 502                                 PNET_BUFFER_LIST *nextNbl)
 503 {
 504     NTSTATUS status = STATUS_SUCCESS;
 505     PNET_BUFFER_LIST newNbls = NULL;
 506     PNET_BUFFER_LIST lastNbl = NULL;
 507     PNET_BUFFER_LIST nbl = NULL;
 508     POVS_BUFFER_CONTEXT bufContext = NULL;
 509     BOOLEAN error = TRUE;
 510
 511     do {
 512         /* Create new NBLs from curNbl with multiple net buffers. */
 513         newNbls = OvsPartialCopyToMultipleNBLs(switchContext,
 514                                                *curNbl, 0, 0, TRUE);
 515         if (NULL == newNbls) {
 516             OVS_LOG_ERROR("Failed to allocate NBLs with single NB.");
 517             status = NDIS_STATUS_RESOURCES;
 518             break;
 519         }
 520
 521         nbl = newNbls;
 522         while (nbl) {
 523             lastNbl = nbl;
 524             nbl = NET_BUFFER_LIST_NEXT_NBL(nbl);
 525         }
 526         lastNbl->Next = *nextNbl;
 527         *nextNbl = newNbls->Next;
 528         *curNbl = newNbls;
 529         (*curNbl)->Next = NULL;
 530
 531         OvsCompleteNBL(switchContext, *curNbl, TRUE);
 532
 533         error = FALSE;
 534     } while (error);
 535
 536     return status;
 537 }