[{"data":1,"prerenderedAt":764},["ShallowReactive",2],{"blog-\u002Fblog\u002Fpython-multiprocessing-explained":3},{"id":4,"title":5,"body":6,"description":750,"difficulty":751,"extension":752,"framework":753,"frameworkSlug":40,"meta":754,"navigation":76,"order":65,"path":755,"qaPath":756,"seo":757,"stem":758,"subtopic":759,"topic":760,"topicSlug":761,"updated":762,"__hash__":763},"blog\u002Fblog\u002Fpython-multiprocessing-explained.md","Python Multiprocessing Explained — Escaping the GIL, Process Pools, and Sharing Data",{"type":7,"value":8,"toc":739},"minimark",[9,14,28,32,35,238,249,257,268,272,283,436,453,457,464,530,533,537,555,684,691,695,707,711,735],[10,11,13],"h2",{"id":12},"python-multiprocessing-explained","Python multiprocessing, explained",[15,16,17,18,22,23,27],"p",{},"The GIL stops Python threads from running CPU-bound code in parallel. ",[19,20,21],"code",{},"multiprocessing","\nsidesteps it entirely by running code in ",[24,25,26],"strong",{},"separate processes",", each with its own\ninterpreter and its own GIL. That buys real parallelism on multiple cores — at the cost of\nnot sharing memory, so data must be explicitly passed around.",[10,29,31],{"id":30},"why-processes-instead-of-threads","Why processes instead of threads",[15,33,34],{},"A thread shares the parent's memory and GIL; a process is an independent OS process with\nits own memory space. For CPU-bound work, processes win because the OS schedules them\nacross cores simultaneously.",[36,37,42],"pre",{"className":38,"code":39,"language":40,"meta":41,"style":41},"language-python shiki shiki-themes github-light github-dark","from multiprocessing import Process\nimport os\n\ndef worker(n):\n    print(f\"worker {n} in pid {os.getpid()}\")\n\nif __name__ == \"__main__\":          # required guard, see below\n    procs = [Process(target=worker, args=(i,)) for i in range(3)]\n    for p in procs: p.start()\n    for p in procs: p.join()        # wait for all to finish\n","python","",[19,43,44,63,71,78,91,133,138,160,209,223],{"__ignoreMap":41},[45,46,49,53,57,60],"span",{"class":47,"line":48},"line",1,[45,50,52],{"class":51},"szBVR","from",[45,54,56],{"class":55},"sVt8B"," multiprocessing ",[45,58,59],{"class":51},"import",[45,61,62],{"class":55}," Process\n",[45,64,66,68],{"class":47,"line":65},2,[45,67,59],{"class":51},[45,69,70],{"class":55}," os\n",[45,72,74],{"class":47,"line":73},3,[45,75,77],{"emptyLinePlaceholder":76},true,"\n",[45,79,81,84,88],{"class":47,"line":80},4,[45,82,83],{"class":51},"def",[45,85,87],{"class":86},"sScJk"," worker",[45,89,90],{"class":55},"(n):\n",[45,92,94,98,101,104,108,111,114,117,120,122,125,127,130],{"class":47,"line":93},5,[45,95,97],{"class":96},"sj4cs","    print",[45,99,100],{"class":55},"(",[45,102,103],{"class":51},"f",[45,105,107],{"class":106},"sZZnC","\"worker ",[45,109,110],{"class":96},"{",[45,112,113],{"class":55},"n",[45,115,116],{"class":96},"}",[45,118,119],{"class":106}," in pid ",[45,121,110],{"class":96},[45,123,124],{"class":55},"os.getpid()",[45,126,116],{"class":96},[45,128,129],{"class":106},"\"",[45,131,132],{"class":55},")\n",[45,134,136],{"class":47,"line":135},6,[45,137,77],{"emptyLinePlaceholder":76},[45,139,141,144,147,150,153,156],{"class":47,"line":140},7,[45,142,143],{"class":51},"if",[45,145,146],{"class":96}," __name__",[45,148,149],{"class":51}," ==",[45,151,152],{"class":106}," \"__main__\"",[45,154,155],{"class":55},":          ",[45,157,159],{"class":158},"sJ8bj","# required guard, see below\n",[45,161,163,166,169,172,176,178,181,184,186,189,192,195,198,201,203,206],{"class":47,"line":162},8,[45,164,165],{"class":55},"    procs ",[45,167,168],{"class":51},"=",[45,170,171],{"class":55}," [Process(",[45,173,175],{"class":174},"s4XuR","target",[45,177,168],{"class":51},[45,179,180],{"class":55},"worker, ",[45,182,183],{"class":174},"args",[45,185,168],{"class":51},[45,187,188],{"class":55},"(i,)) ",[45,190,191],{"class":51},"for",[45,193,194],{"class":55}," i ",[45,196,197],{"class":51},"in",[45,199,200],{"class":96}," range",[45,202,100],{"class":55},[45,204,205],{"class":96},"3",[45,207,208],{"class":55},")]\n",[45,210,212,215,218,220],{"class":47,"line":211},9,[45,213,214],{"class":51},"    for",[45,216,217],{"class":55}," p ",[45,219,197],{"class":51},[45,221,222],{"class":55}," procs: p.start()\n",[45,224,226,228,230,232,235],{"class":47,"line":225},10,[45,227,214],{"class":51},[45,229,217],{"class":55},[45,231,197],{"class":51},[45,233,234],{"class":55}," procs: p.join()        ",[45,236,237],{"class":158},"# wait for all to finish\n",[15,239,240,241,244,245,248],{},"Each ",[19,242,243],{},"worker"," prints a ",[24,246,247],{},"different"," PID — these are genuinely separate processes running\nin parallel.",[10,250,252,253,256],{"id":251},"the-main-guard-is-mandatory","The ",[24,254,255],{},"main"," guard is mandatory",[15,258,259,260,263,264,267],{},"On Windows and macOS the default start method is ",[24,261,262],{},"spawn",": a fresh interpreter is launched\nthat re-imports your module. Without ",[19,265,266],{},"if __name__ == \"__main__\":",", that re-import would spawn\nprocesses recursively. Always guard the entry point — this is the single most common\nmultiprocessing bug.",[10,269,271],{"id":270},"pool-is-the-workhorse","Pool is the workhorse",[15,273,274,275,278,279,282],{},"For \"apply this function to many inputs,\" ",[19,276,277],{},"Pool"," manages a fixed set of worker processes and\ndistributes the work. ",[19,280,281],{},"map"," returns results in input order.",[36,284,286],{"className":38,"code":285,"language":40,"meta":41,"style":41},"from multiprocessing import Pool\n\ndef heavy(n):\n    return sum(i * i for i in range(n))\n\nif __name__ == \"__main__\":\n    with Pool(processes=4) as pool:\n        results = pool.map(heavy, [10**6, 2*10**6, 3*10**6])\n    print(results)\n",[19,287,288,299,303,312,339,343,356,381,429],{"__ignoreMap":41},[45,289,290,292,294,296],{"class":47,"line":48},[45,291,52],{"class":51},[45,293,56],{"class":55},[45,295,59],{"class":51},[45,297,298],{"class":55}," Pool\n",[45,300,301],{"class":47,"line":65},[45,302,77],{"emptyLinePlaceholder":76},[45,304,305,307,310],{"class":47,"line":73},[45,306,83],{"class":51},[45,308,309],{"class":86}," heavy",[45,311,90],{"class":55},[45,313,314,317,320,323,326,328,330,332,334,336],{"class":47,"line":80},[45,315,316],{"class":51},"    return",[45,318,319],{"class":96}," sum",[45,321,322],{"class":55},"(i ",[45,324,325],{"class":51},"*",[45,327,194],{"class":55},[45,329,191],{"class":51},[45,331,194],{"class":55},[45,333,197],{"class":51},[45,335,200],{"class":96},[45,337,338],{"class":55},"(n))\n",[45,340,341],{"class":47,"line":93},[45,342,77],{"emptyLinePlaceholder":76},[45,344,345,347,349,351,353],{"class":47,"line":135},[45,346,143],{"class":51},[45,348,146],{"class":96},[45,350,149],{"class":51},[45,352,152],{"class":106},[45,354,355],{"class":55},":\n",[45,357,358,361,364,367,369,372,375,378],{"class":47,"line":140},[45,359,360],{"class":51},"    with",[45,362,363],{"class":55}," Pool(",[45,365,366],{"class":174},"processes",[45,368,168],{"class":51},[45,370,371],{"class":96},"4",[45,373,374],{"class":55},") ",[45,376,377],{"class":51},"as",[45,379,380],{"class":55}," pool:\n",[45,382,383,386,388,391,394,397,400,403,406,408,410,412,414,416,418,420,422,424,426],{"class":47,"line":162},[45,384,385],{"class":55},"        results ",[45,387,168],{"class":51},[45,389,390],{"class":55}," pool.map(heavy, [",[45,392,393],{"class":96},"10",[45,395,396],{"class":51},"**",[45,398,399],{"class":96},"6",[45,401,402],{"class":55},", ",[45,404,405],{"class":96},"2",[45,407,325],{"class":51},[45,409,393],{"class":96},[45,411,396],{"class":51},[45,413,399],{"class":96},[45,415,402],{"class":55},[45,417,205],{"class":96},[45,419,325],{"class":51},[45,421,393],{"class":96},[45,423,396],{"class":51},[45,425,399],{"class":96},[45,427,428],{"class":55},"])\n",[45,430,431,433],{"class":47,"line":211},[45,432,97],{"class":96},[45,434,435],{"class":55},"(results)\n",[15,437,438,440,441,444,445,448,449,452],{},[19,439,277],{}," also offers ",[19,442,443],{},"imap"," (lazy, ordered), ",[19,446,447],{},"imap_unordered"," (yields as they finish), and\n",[19,450,451],{},"apply_async"," for single calls with callbacks.",[10,454,456],{"id":455},"everything-crosses-the-boundary-via-pickling","Everything crosses the boundary via pickling",[15,458,459,460,463],{},"Because processes don't share memory, arguments and return values are ",[24,461,462],{},"pickled",", sent\nthrough a pipe, and unpickled on the other side. This has consequences:",[36,465,467],{"className":38,"code":466,"language":40,"meta":41,"style":41},"# Fails — lambdas and local functions can't be pickled\npool.map(lambda n: n*2, data)        # PicklingError\n\n# Works — top-level functions are picklable\ndef double(n): return n * 2\npool.map(double, data)\n",[19,468,469,474,495,499,504,525],{"__ignoreMap":41},[45,470,471],{"class":47,"line":48},[45,472,473],{"class":158},"# Fails — lambdas and local functions can't be pickled\n",[45,475,476,479,482,485,487,489,492],{"class":47,"line":65},[45,477,478],{"class":55},"pool.map(",[45,480,481],{"class":51},"lambda",[45,483,484],{"class":55}," n: n",[45,486,325],{"class":51},[45,488,405],{"class":96},[45,490,491],{"class":55},", data)        ",[45,493,494],{"class":158},"# PicklingError\n",[45,496,497],{"class":47,"line":73},[45,498,77],{"emptyLinePlaceholder":76},[45,500,501],{"class":47,"line":80},[45,502,503],{"class":158},"# Works — top-level functions are picklable\n",[45,505,506,508,511,514,517,520,522],{"class":47,"line":93},[45,507,83],{"class":51},[45,509,510],{"class":86}," double",[45,512,513],{"class":55},"(n): ",[45,515,516],{"class":51},"return",[45,518,519],{"class":55}," n ",[45,521,325],{"class":51},[45,523,524],{"class":96}," 2\n",[45,526,527],{"class":47,"line":135},[45,528,529],{"class":55},"pool.map(double, data)\n",[15,531,532],{},"Pickling also means large arguments are expensive to send, and the target function must be\nimportable at top level. Keep data passed to workers small.",[10,534,536],{"id":535},"sharing-data-queues-and-shared-memory","Sharing data: queues and shared memory",[15,538,539,540,543,544,547,548,547,551,554],{},"When workers need to communicate, use a process-safe ",[19,541,542],{},"Queue"," for message passing, or\n",[19,545,546],{},"shared_memory"," \u002F ",[19,549,550],{},"Value",[19,552,553],{},"Array"," for shared state.",[36,556,558],{"className":38,"code":557,"language":40,"meta":41,"style":41},"from multiprocessing import Process, Queue\n\ndef producer(q):\n    for i in range(5):\n        q.put(i)\n\nif __name__ == \"__main__\":\n    q = Queue()\n    p = Process(target=producer, args=(q,))\n    p.start(); p.join()\n    while not q.empty():\n        print(q.get())\n",[19,559,560,571,575,585,603,608,612,624,634,658,663,675],{"__ignoreMap":41},[45,561,562,564,566,568],{"class":47,"line":48},[45,563,52],{"class":51},[45,565,56],{"class":55},[45,567,59],{"class":51},[45,569,570],{"class":55}," Process, Queue\n",[45,572,573],{"class":47,"line":65},[45,574,77],{"emptyLinePlaceholder":76},[45,576,577,579,582],{"class":47,"line":73},[45,578,83],{"class":51},[45,580,581],{"class":86}," producer",[45,583,584],{"class":55},"(q):\n",[45,586,587,589,591,593,595,597,600],{"class":47,"line":80},[45,588,214],{"class":51},[45,590,194],{"class":55},[45,592,197],{"class":51},[45,594,200],{"class":96},[45,596,100],{"class":55},[45,598,599],{"class":96},"5",[45,601,602],{"class":55},"):\n",[45,604,605],{"class":47,"line":93},[45,606,607],{"class":55},"        q.put(i)\n",[45,609,610],{"class":47,"line":135},[45,611,77],{"emptyLinePlaceholder":76},[45,613,614,616,618,620,622],{"class":47,"line":140},[45,615,143],{"class":51},[45,617,146],{"class":96},[45,619,149],{"class":51},[45,621,152],{"class":106},[45,623,355],{"class":55},[45,625,626,629,631],{"class":47,"line":162},[45,627,628],{"class":55},"    q ",[45,630,168],{"class":51},[45,632,633],{"class":55}," Queue()\n",[45,635,636,639,641,644,646,648,651,653,655],{"class":47,"line":211},[45,637,638],{"class":55},"    p ",[45,640,168],{"class":51},[45,642,643],{"class":55}," Process(",[45,645,175],{"class":174},[45,647,168],{"class":51},[45,649,650],{"class":55},"producer, ",[45,652,183],{"class":174},[45,654,168],{"class":51},[45,656,657],{"class":55},"(q,))\n",[45,659,660],{"class":47,"line":225},[45,661,662],{"class":55},"    p.start(); p.join()\n",[45,664,666,669,672],{"class":47,"line":665},11,[45,667,668],{"class":51},"    while",[45,670,671],{"class":51}," not",[45,673,674],{"class":55}," q.empty():\n",[45,676,678,681],{"class":47,"line":677},12,[45,679,680],{"class":96},"        print",[45,682,683],{"class":55},"(q.get())\n",[15,685,686,687,690],{},"For numeric arrays shared without copying, ",[19,688,689],{},"multiprocessing.shared_memory.SharedMemory","\n(3.8+) lets processes view the same buffer — ideal for large NumPy arrays.",[10,692,694],{"id":693},"when-multiprocessing-is-worth-it","When multiprocessing is worth it",[15,696,697,698,701,702,706],{},"It pays off for ",[24,699,700],{},"CPU-bound"," tasks big enough to dwarf the process startup and pickling\noverhead: image processing, simulations, number crunching. For I\u002FO-bound work, threads or\nasyncio are lighter. And for short tasks, the overhead of spawning processes and shipping\ndata can make multiprocessing ",[703,704,705],"em",{},"slower"," than a plain loop.",[10,708,710],{"id":709},"recap","Recap",[15,712,713,715,716,718,719,721,722,725,726,728,729,731,732,734],{},[19,714,21],{}," achieves true parallelism by running code in ",[24,717,26],{},", each\nwith its own interpreter and GIL — the way around the GIL for CPU-bound work. Always wrap\nthe entry point in ",[19,720,266],{}," because of the spawn start method. Use\n",[19,723,724],{},"Pool.map"," to fan work across workers, remember that all arguments and results are\n",[24,727,462],{}," (so no lambdas, keep payloads small), and share state through ",[19,730,542],{}," or\n",[19,733,546],{},". Reach for it when CPU work outweighs the startup and serialisation cost.",[736,737,738],"style",{},"html pre.shiki code .szBVR, html code.shiki .szBVR{--shiki-default:#D73A49;--shiki-dark:#F97583}html pre.shiki code .sVt8B, html code.shiki .sVt8B{--shiki-default:#24292E;--shiki-dark:#E1E4E8}html pre.shiki code .sScJk, html code.shiki .sScJk{--shiki-default:#6F42C1;--shiki-dark:#B392F0}html pre.shiki code .sj4cs, html code.shiki .sj4cs{--shiki-default:#005CC5;--shiki-dark:#79B8FF}html pre.shiki code .sZZnC, html code.shiki .sZZnC{--shiki-default:#032F62;--shiki-dark:#9ECBFF}html pre.shiki code .sJ8bj, html code.shiki .sJ8bj{--shiki-default:#6A737D;--shiki-dark:#6A737D}html pre.shiki code .s4XuR, html code.shiki .s4XuR{--shiki-default:#E36209;--shiki-dark:#FFAB70}html .default .shiki span {color: var(--shiki-default);background: var(--shiki-default-bg);font-style: var(--shiki-default-font-style);font-weight: var(--shiki-default-font-weight);text-decoration: var(--shiki-default-text-decoration);}html .shiki span {color: var(--shiki-default);background: var(--shiki-default-bg);font-style: var(--shiki-default-font-style);font-weight: var(--shiki-default-font-weight);text-decoration: var(--shiki-default-text-decoration);}html .dark .shiki span {color: var(--shiki-dark);background: var(--shiki-dark-bg);font-style: var(--shiki-dark-font-style);font-weight: var(--shiki-dark-font-weight);text-decoration: var(--shiki-dark-text-decoration);}html.dark .shiki span {color: var(--shiki-dark);background: var(--shiki-dark-bg);font-style: var(--shiki-dark-font-style);font-weight: var(--shiki-dark-font-weight);text-decoration: var(--shiki-dark-text-decoration);}",{"title":41,"searchDepth":65,"depth":65,"links":740},[741,742,743,745,746,747,748,749],{"id":12,"depth":65,"text":13},{"id":30,"depth":65,"text":31},{"id":251,"depth":65,"text":744},"The main guard is mandatory",{"id":270,"depth":65,"text":271},{"id":455,"depth":65,"text":456},{"id":535,"depth":65,"text":536},{"id":693,"depth":65,"text":694},{"id":709,"depth":65,"text":710},"How Python multiprocessing achieves true parallelism by sidestepping the GIL — spawning processes, using Pool, passing data with pickling, and sharing state with queues and shared memory.","hard","md","Python",{},"\u002Fblog\u002Fpython-multiprocessing-explained","\u002Fpython\u002Fconcurrency\u002Fmultiprocessing",{"title":5,"description":750},"blog\u002Fpython-multiprocessing-explained","Multiprocessing","Concurrency & Parallelism","concurrency","2026-06-19","v5IJqD2z7DPKD6ljdorUeA1Ch8wDFmQkgqqTfp3DDrQ",1782244092988]